﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI.WebControls;
using CrawlerCore;
using WCrawlerLib;
using HtmlAgilityPack;
using System.Data;
using System.Threading;
using System.Threading.Tasks;

namespace WCrawler.XP
{
    public partial class SearchGoogle : System.Web.UI.Page
    {
        private int _index = 0;

        protected void Page_Load(object sender, EventArgs e)
        {
            if (!IsPostBack)
            {
            }
        }

        public DataTable CreateDataTable()
        {
            DataTable myTable;

            // Create a new DataTable.
            myTable = new DataTable("XPathResult");

            // Create DataColumn objects of data types.
            DataColumn col = new DataColumn("Url");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            //Create primary key
            myTable.PrimaryKey = new DataColumn[]{col};

            col = new DataColumn("OriginalUrl");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("Description");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("ForumTypeId");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("Type");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("Version");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("LocationId");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("Location");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("FlagUrl");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("Rank");
            col.DataType = System.Type.GetType("System.String");
            myTable.Columns.Add(col);
            col = new DataColumn("IsExist");
            col.DataType = System.Type.GetType("System.Boolean");
            myTable.Columns.Add(col);
            
            return myTable;
        }

        public Dictionary<string, string> GetItems(HtmlDocument doc, string[] xpath)
        {
            Dictionary<string, string> result = new Dictionary<string, string>();
            HtmlNodeCollection nodeCollection = doc.DocumentNode.SelectNodes(xpath[3]); //".//div[@class='vsc']";
            if (nodeCollection != null)
            {
                foreach (HtmlNode node in nodeCollection)
                {
                    string href = node.SelectNodes(xpath[4])[0].Attributes["href"].Value; //".//a[@class='l noline' or @class='l']"
                    string des = node.SelectNodes(xpath[5])[0].InnerHtml; //".//span[@class='st']"
                    result.Add(href, des);
                }
            }
            return result;
        }

        public void StartLoadPage(int pageIndex, string url, bool isSearchPaging)
        {
            string[] arrayParams = (string[])Session["arrayParams"];
            BrowserSession_General br = new BrowserSession_General();
            HtmlAgilityPack.HtmlDocument doc = br.Get(url);
            //Get items of first page
            Dictionary<string, string> result = GetItems(doc, arrayParams);
            Dictionary<string, string> globalResults = (Dictionary<string, string>)Session["globalResults"];
            
            foreach (KeyValuePair<string, string> value in result)
            {
                if (!globalResults.Keys.Contains(value.Key))
                {
                    globalResults.Add(value.Key, value.Value);
                }
            }

            //global page url
            Dictionary<int, string> globalPageUrls = (Dictionary<int, string>)Session["globalPageUrls"];

            if (isSearchPaging)
            {
                //Get paging indexes
                int index = -1;
                //local page url
                Dictionary<int, string> localPageUrls = new Dictionary<int, string>();

                string xpath = arrayParams[2]; //get paging indexes
                HtmlNodeCollection nodeCollection = doc.DocumentNode.SelectNodes(xpath);
                if (nodeCollection != null)
                {
                    //================================================================
                    //Get all paging indexes in current page
                    foreach (HtmlNode node in nodeCollection)
                    {
                        //exclude page with max index to afterthat load with StartLoadPage function at the last code row
                        if (node.Attributes["href"] != null)
                        {
                            string href = node.Attributes["href"].Value;
                            //Get index number
                            HtmlNodeCollection indexNodeCollect = node.SelectNodes(".//text()");
                            if (indexNodeCollect != null)
                            {
                                foreach (HtmlNode indexNode in indexNodeCollect)
                                {
                                    if (!string.IsNullOrEmpty(indexNode.InnerHtml.Trim()))
                                    {
                                        try
                                        {
                                            //cast into number
                                            index = int.Parse(indexNode.InnerHtml.Trim());

                                            //if index not save in globalPageUrls then add new
                                            if (!globalPageUrls.Keys.Contains(index))
                                            {
                                                localPageUrls.Add(index, href);
                                                globalPageUrls.Add(index, href);
                                            }
                                        }
                                        catch
                                        {

                                        }
                                    }
                                }
                            }
                        }
                    }

                    //================================================================
                    //Load all paging (exclude max index of paging) with task parallel 
                    object objLock = new object();
                    int[] keys = localPageUrls.Keys.ToArray();
                    //Notes: keys.Length - 1 to exclude lastest index
                    Parallel.For(0, keys.Length - 1, i =>
                    {
                        Thread.Sleep(i * 2000);
                        string indexUrl = arrayParams[0] + localPageUrls[keys[i]];
                        HtmlDocument indexDoc = br.Get(indexUrl);
                        Dictionary<string, string> result1 = GetItems(indexDoc, arrayParams);
                        foreach (KeyValuePair<string, string> value in result1)
                        {
                            lock (objLock)
                            {
                                if (!globalResults.Keys.Contains(value.Key))
                                {
                                    globalResults.Add(value.Key, value.Value);
                                }
                            }
                        }
                    });
                    Thread.Sleep(2000);

                    //================================================================
                    //Save DataTable into session
                    Session["globalResults"] = globalResults;
                    //Save globalPageUrls
                    Session["globalPageUrls"] = globalPageUrls;
                    //Load paging with max index
                    if (localPageUrls.Keys.Count > 0)
                    {
                        pageIndex = keys[localPageUrls.Keys.Count - 1];
                        url = arrayParams[0] + localPageUrls[pageIndex];
                        StartLoadPage(pageIndex, url, isSearchPaging);
                    }
                }
                else
                {
                    //Save DataTable into session
                    Session["globalResults"] = globalResults;
                    //Save globalPageUrls
                    Session["globalPageUrls"] = globalPageUrls;

                    pageIndex -= 1;
                    url = arrayParams[0] + globalPageUrls[pageIndex];
                    StartLoadPage(pageIndex, url, isSearchPaging);
                }
            }
            else
            {
                //Save DataTable into session
                Session["globalResults"] = globalResults;
                //Save globalPageUrls
                Session["globalPageUrls"] = globalPageUrls;
            }
        }

        public string[] GetSearchEngine(int index)
        {
            string[] values = new string[6];
            switch (index){
                case 1:
                    values[0] = "http://www.google.com"; //main page
                    values[1] = "/search?hl=en&q=[text]"; //start search engine
                    values[2] = ".//table[@id='nav']//a"; //get paging indexes
                    values[3] = ".//div[@class='vsc']"; //get item with url and description
                    values[4] = ".//a[@class='l noline' or @class='l']"; //get url
                    values[5] = ".//span[@class='st']"; //get description
                    break;
                case 2:
                    values[0] = "http://www.bing.com";
                    values[1] = "/search?q=[text]&go=&qs=n&sk=&form=QBLH";
                    values[2] = ".//div[@class='sb_pag']//a";
                    values[3] = ".//div[@class='sa_cc']";
                    values[4] = "./div[1]//a";
                    values[5] = ".//p";
                    break;
                case 3:
                    values[0] = "http://search.yahoo.com";
                    values[1] = "/search?vc=&p=[text]&toggle=1&cop=mss&ei=UTF-8&fr=yfp-t-738";
                    values[2] = ".//div[@id='pg']//a";
                    values[3] = ".//div[@id='web']/ol/li";
                    values[4] = ".//a[@class='yschttl spt']";
                    values[5] = ".//div[@class='abstr' or @class='sm-abs']";
                    break;
                default:
                    values[0] = "http://www.google.com"; //main page
                    values[1] = "/search?hl=en&q=[text]"; //start search engine
                    values[2] = ".//table[@id='nav']//a"; //get paging indexes
                    values[3] = ".//div[@class='vsc']"; //get item with url and description
                    values[4] = ".//a[@class='l noline' or @class='l']"; //get url
                    values[5] = ".//span[@class='st']"; //get description
                    break;
            }
            return values;
        }

        protected void btnSearch_Click(object sender, EventArgs e)
        {
            //string url = "http://www.google.com/";
            //br.Get(url);
            //url = "http://www.google.com/preferences?hl=en";
            //br.Get(url);
            //url = "http://www.google.com/setprefs?sig=[sig]&hl=en&safeui=images&suggon=2&num=100&q=&submit2=Save+Preferences".Replace("[sig]", br.GoogleSIG);
            //br.Get(url);
            //string url = "http://www.google.com.vn/search?hl=vi&source=hp&q=Internet+Download+Manager+6.07+Final&oq=Internet+Download+Manager+6.07+Final&aq=f&aqi=g1&aql=&gs_sm=si&gs_upl=19149l19149l0l22395l1l1l0l0l0l0l955l955l6-1l1l0&biw=1280&bih=716&cad=cbv";

            string[] arrayParams = GetSearchEngine(int.Parse(radioButtonListSearchEngines.SelectedValue));
            Session["arrayParams"] = arrayParams;
            string url = arrayParams[0] + arrayParams[1].Replace("[text]", HttpUtility.UrlEncode(txtSearchString.Text));
            
            Dictionary<int, string> globalPageUrls = new Dictionary<int, string>();
            globalPageUrls.Add(1, url);
            Session["globalPageUrls"] = globalPageUrls;
            //global items
            Dictionary<string, string> globalResults = new Dictionary<string, string>();
            Session["globalResults"] = globalResults;

            //=============================================================================
            //Start load page to get urls
            StartLoadPage(1, url, chkSearchPaging.Checked);
            //=============================================================================
            
            //Get final result
            globalResults = (Dictionary<string, string>)Session["globalResults"];

            //Filter result by get formatted url
            Dictionary<string, string> filterResults = new Dictionary<string, string>();
            DataTable myTable = CreateDataTable();
            foreach (KeyValuePair<string, string> value in globalResults)
            {
                string webUrl = GeneralClass.FormatUrl2(value.Key);
                if (!filterResults.Keys.Contains(webUrl))
                {
                    filterResults.Add(webUrl, value.Value);
                    //Check forum url is exist in database
                    string description = value.Value;
                    bool isExist = false;
                    Forum obj = new Forum();
                    if (obj.IsExists(webUrl))
                    {
                        description = "<p style=\"color:#FF0000; \"> ++++++ THE PAGE IS EXIST ++++++ </p><br/>" + description;
                        isExist = true;
                    }

                    DataRow myNewRow = myTable.NewRow();
                    myNewRow["Url"] = webUrl;
                    myNewRow["OriginalUrl"] = value.Key;
                    myNewRow["Description"] = description;
                    myNewRow["IsExist"] = isExist;
                    myTable.Rows.Add(myNewRow);
                }
            }
            Session["myTable"] = myTable;
            dropDownListView.SelectedIndex = 0;
            dropDownListView_SelectedIndexChanged(null, null);
        }

        public void GridViewDataBind()
        {

        }
        protected void btnCheckInformation_Click(object sender, EventArgs e)
        {
            if (Session["myTable"] != null)
            {
                DataTable myTable = (DataTable)Session["myTable"];
                Parallel.ForEach<GridViewRow>(gridViewResult.Rows.OfType<GridViewRow>(), row =>
                {
                    CheckBox chkBx = (CheckBox)row.FindControl("chkCheckItem");
                    if (chkBx != null && chkBx.Checked)
                    {
                        string forumUrl = ((HyperLink)row.FindControl("hyperLinkUrl")).Text;
                        DataRow dtRow = myTable.Rows.Find(forumUrl);

                        Forum objForum = new Forum();
                        objForum = objForum.CheckForumInfoBySearchEngine(forumUrl);
                        if (objForum.ForumId == -1)
                        {
                            if (objForum.ForumTypeId.HasValue)
                            {
                                ForumType obj = new ForumType();
                                obj = obj.GetForumType(objForum.ForumTypeId.Value);
                                if (obj != null)
                                {
                                    dtRow["ForumTypeId"] = obj.ForumTypeId;
                                    dtRow["Type"] = obj.TypeName;
                                    dtRow["Version"] = objForum.Version;
                                }
                            }
                            if (objForum.LocationId.HasValue)
                            {
                                Location obj = new Location();
                                obj = obj.GetLocation(objForum.LocationId.Value);
                                if (obj != null)
                                {
                                    dtRow["LocationId"] = obj.LocationId;
                                    dtRow["Location"] = obj.LocationName;
                                    dtRow["FlagUrl"] = obj.FlagUrl;
                                }
                            }
                            if (objForum.AlexaRank.HasValue)
                                dtRow["Rank"] = objForum.AlexaRank.Value.ToString();
                        }
                    }
                });
                Session["myTable"] = myTable;
                dropDownListView_SelectedIndexChanged(null, null);
            }
        }

        protected void gridViewResult_RowDataBound(object sender, GridViewRowEventArgs e)
        {
            if (e.Row.RowType == DataControlRowType.Header)
                _index = 1;
            else if (e.Row.RowType == DataControlRowType.DataRow)
            {
                //Create order numbers
                ((Label)e.Row.FindControl("lblNum")).Text = (_index++).ToString();
                Image img = (Image)e.Row.FindControl("imgFlag");
                if (img.ImageUrl == "")
                    img.Visible = false;
            }
        }

        protected void dropDownListView_SelectedIndexChanged(object sender, EventArgs e)
        {
            if (Session["myTable"] != null)
            {
                DataTable myTable = (DataTable)Session["myTable"];
                //Default select all rows
                EnumerableRowCollection<DataRow> query;
                if (dropDownListView.SelectedIndex == 1) //available
                {
                    query = from p in myTable.AsEnumerable()
                            where p.Field<bool>("IsExist") == true
                            select p;
                }
                else if (dropDownListView.SelectedIndex == 2) //Not available
                {
                    query = from p in myTable.AsEnumerable()
                            where p.Field<bool>("IsExist") == false
                            select p;
                }
                else //default select all rows
                {
                    query = from p in myTable.AsEnumerable()
                            select p;
                }
                gridViewResult.DataSource = query.AsDataView();
                gridViewResult.DataBind();
            }
        }

        protected void btnAddSelectedForums_Click(object sender, EventArgs e)
        {
            //foreach (GridViewRow row in gridViewResult.Rows)
            //{
            //    CheckBox chkBx = (CheckBox)row.FindControl("chkCheckItem");
            //    if (chkBx.Checked)
            //    {
            //        string forumUrl = ((Label)row.FindControl("lblUrl")).Text;
            //        Forum objForum = new Forum();
            //        objForum.AddForum(forumUrl);
            //    }
            //}

            if (Session["myTable"] != null)
            {
                DataTable myTable = (DataTable)Session["myTable"];
                Parallel.ForEach<GridViewRow>(gridViewResult.Rows.OfType<GridViewRow>(), row =>
                //Parallel.For(0, gridViewResult.Rows.Count, i =>
                {
                    CheckBox chkBx = (CheckBox)row.FindControl("chkCheckItem");
                    if (chkBx != null && chkBx.Checked)
                    {
                        string forumUrl = ((HyperLink)row.FindControl("hyperLinkUrl")).Text;
                        DataRow dtRow = myTable.Rows.Find(forumUrl);

                        Forum objForum = new Forum();
                        //if add successful
                        objForum.AddForum(forumUrl, 2, dtRow["LocationId"].ToString(), dtRow["Rank"].ToString(), dtRow["ForumTypeId"].ToString(), dtRow["Version"].ToString());
                        //if (objForum.AddForum(forumUrl, 2))
                        //{
                        //    string description = dtRow["Description"].ToString();
                        //    description = "<p style=\"color:#FF0000; \"> ++++++ THE PAGE IS EXIST ++++++ </p><br/>" + description;
                        //    dtRow["IsExist"] = true;
                        //}
                    }
                });
                Session["myTable"] = myTable;
                dropDownListView_SelectedIndexChanged(null, null);
            }
            
        }
    }
}